{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# -*- encoding:utf-8 -*-\n",
    "import codecs\n",
    "\n",
    "import jieba\n",
    "import jieba.analyse\n",
    "import jieba.posseg\n",
    "import pandas as pd\n",
    "\n",
    "import param\n",
    "import util\n",
    "\n",
    "\n",
    "############################ 定义分词函数 ############################\n",
    "def split_word(text, stopwords):\n",
    "    word_list = jieba.cut(text)\n",
    "    start = True\n",
    "    result = ''\n",
    "    for word in word_list:\n",
    "        word = word.strip()\n",
    "        if word not in stopwords:\n",
    "            if start:\n",
    "                result = word\n",
    "                start = False\n",
    "            else:\n",
    "                result += ' ' + word\n",
    "    return result.encode('utf-8')\n",
    "\n",
    "############################ 加载停用词 ############################\n",
    "stopwords = {}\n",
    "for line in codecs.open(param.data_path + '/input/stop.txt', 'r', 'utf-8'):\n",
    "    stopwords[line.rstrip()] = 1\n",
    "\n",
    "############################ 加载数据 & 分词 ############################\n",
    "df_tr = []\n",
    "for i,line in enumerate(open(param.data_path + '/input/train.txt')):\n",
    "    if i % 1000 == 1:\n",
    "        util.log('iter = %d' % i)\n",
    "    segs = line.split('\\t')\n",
    "    row = {}\n",
    "    row['id'] = segs[0]\n",
    "    row['content'] = split_word(segs[1].strip(), stopwords)\n",
    "    row['penalty'] = segs[2]\n",
    "    row['laws'] = segs[3].strip()\n",
    "    df_tr.append(row)\n",
    "df_tr = pd.DataFrame(df_tr)\n",
    "\n",
    "df_te = []\n",
    "for i,line in enumerate(open(param.data_path + '/input/test.txt')):\n",
    "    if i % 1000 == 1:\n",
    "        util.log('iter = %d' % i)\n",
    "    segs = line.split('\\t')\n",
    "    row = {}\n",
    "    row['id'] = segs[0]\n",
    "    row['content'] = split_word(segs[1].strip(), stopwords)\n",
    "    df_te.append(row)\n",
    "df_te = pd.DataFrame(df_te)\n",
    "\n",
    "print(df_tr.shape)\n",
    "print(df_te.shape)\n",
    "\n",
    "############################ 写出数据 ############################\n",
    "df_all = pd.concat([df_tr, df_te]).fillna(0)\n",
    "df_all.to_csv(param.data_path + '/output/corpus/all_data.csv', index=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# coding=utf-8\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from sklearn.cross_validation import StratifiedKFold\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.metrics import f1_score\n",
    "\n",
    "import param\n",
    "import util\n",
    "\n",
    "\n",
    "############################ 定义评估函数 ############################\n",
    "def micro_avg_f1(y_true, y_pred):\n",
    "    return f1_score(y_true, y_pred, average='micro')\n",
    "\n",
    "\n",
    "############################ 加载数据 ############################\n",
    "df_all = pd.read_csv(param.data_path + '/output/corpus/all_data.csv', encoding='utf8', nrows=param.train_num)\n",
    "df_all['penalty'] = df_all['penalty'] - 1\n",
    "\n",
    "############################ tfidf ############################\n",
    "tfv = TfidfVectorizer(min_df=3, max_df=0.95, sublinear_tf=True)\n",
    "x_sp = tfv.fit_transform(df_all['content'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "############################ lr stack ############################\n",
    "tr_num = param.cv_train_num\n",
    "num_class = len(pd.value_counts(df_all['penalty']))\n",
    "n = 5\n",
    "\n",
    "x = x_sp[:tr_num]\n",
    "y = df_all['penalty'][:tr_num]\n",
    "x_te = x_sp[tr_num:]\n",
    "y_te = df_all['penalty'][tr_num:]\n",
    "\n",
    "stack = np.zeros((x.shape[0], num_class))\n",
    "stack_te = np.zeros((x_te.shape[0], num_class))\n",
    "\n",
    "score_va = 0\n",
    "score_te = 0\n",
    "for i, (tr, va) in enumerate(StratifiedKFold(y, n_folds=n, random_state=param.seed)):\n",
    "    util.log('stack:%d/%d' % ((i + 1), n))\n",
    "    clf = LogisticRegression(C=2)\n",
    "    clf.fit(x[tr], y[tr])\n",
    "    y_pred_va = clf.predict_proba(x[va])\n",
    "    y_pred_te = clf.predict_proba(x_te)\n",
    "    util.log('va acc:%f' % micro_avg_f1(y[va], clf.predict(x[va])))\n",
    "    util.log('te acc:%f' % micro_avg_f1(y_te, clf.predict(x_te)))\n",
    "    score_va += micro_avg_f1(y[va], clf.predict(x[va]))\n",
    "    score_te += micro_avg_f1(y_te, clf.predict(x_te))\n",
    "    stack[va] += y_pred_va\n",
    "    stack_te += y_pred_te\n",
    "score_va /= n\n",
    "score_te /= n\n",
    "util.log('va avg acc:%f' % score_va)\n",
    "util.log('te avg acc:%f' % score_te)\n",
    "stack_te /= n\n",
    "stack_all = np.vstack([stack, stack_te])\n",
    "df_stack = pd.DataFrame(index=range(len(df_all)))\n",
    "for i in range(stack_all.shape[1]):\n",
    "    df_stack['tfidf_lr_{}'.format(i)] = stack_all[:, i]\n",
    "\n",
    "df_stack.to_csv(param.data_path + '/output/feature/tfidf/lr_prob_12w.csv', index=None, encoding='utf8')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.naive_bayes import BernoulliNB\n",
    "\n",
    "\n",
    "############################ bnb stack ############################\n",
    "tr_num = param.cv_train_num\n",
    "num_class = len(pd.value_counts(df_all['penalty']))\n",
    "n = 5\n",
    "\n",
    "x = x_sp[:tr_num]\n",
    "y = df_all['penalty'][:tr_num]\n",
    "x_te = x_sp[tr_num:]\n",
    "y_te = df_all['penalty'][tr_num:]\n",
    "\n",
    "stack = np.zeros((x.shape[0], num_class))\n",
    "stack_te = np.zeros((x_te.shape[0], num_class))\n",
    "\n",
    "score_va = 0\n",
    "score_te = 0\n",
    "for i, (tr, va) in enumerate(StratifiedKFold(y, n_folds=n, random_state=param.seed)):\n",
    "    util.log('stack:%d/%d' % ((i + 1), n))\n",
    "    clf = BernoulliNB()\n",
    "    clf.fit(x[tr], y[tr])\n",
    "    y_pred_va = clf.predict_proba(x[va])\n",
    "    y_pred_te = clf.predict_proba(x_te)\n",
    "    util.log('va acc:%f' % micro_avg_f1(y[va], clf.predict(x[va])))\n",
    "    util.log('te acc:%f' % micro_avg_f1(y_te, clf.predict(x_te)))\n",
    "    score_va += micro_avg_f1(y[va], clf.predict(x[va]))\n",
    "    score_te += micro_avg_f1(y_te, clf.predict(x_te))\n",
    "    stack[va] += y_pred_va\n",
    "    stack_te += y_pred_te\n",
    "score_va /= n\n",
    "score_te /= n\n",
    "util.log('va avg acc:%f' % score_va)\n",
    "util.log('te avg acc:%f' % score_te)\n",
    "stack_te /= n\n",
    "stack_all = np.vstack([stack, stack_te])\n",
    "df_stack = pd.DataFrame(index=range(len(df_all)))\n",
    "for i in range(stack_all.shape[1]):\n",
    "    df_stack['tfidf_bnb_{}'.format(i)] = stack_all[:, i]\n",
    "\n",
    "df_stack.to_csv(param.data_path + '/output/feature/tfidf/bnb_prob_12w.csv', index=None, encoding='utf8')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.naive_bayes import MultinomialNB\n",
    "\n",
    "\n",
    "############################ mnb stack ############################\n",
    "tr_num = param.cv_train_num\n",
    "num_class = len(pd.value_counts(df_all['penalty']))\n",
    "n = 5\n",
    "\n",
    "x = x_sp[:tr_num]\n",
    "y = df_all['penalty'][:tr_num]\n",
    "x_te = x_sp[tr_num:]\n",
    "y_te = df_all['penalty'][tr_num:]\n",
    "\n",
    "stack = np.zeros((x.shape[0], num_class))\n",
    "stack_te = np.zeros((x_te.shape[0], num_class))\n",
    "\n",
    "score_va = 0\n",
    "score_te = 0\n",
    "for i, (tr, va) in enumerate(StratifiedKFold(y, n_folds=n, random_state=param.seed)):\n",
    "    util.log('stack:%d/%d' % ((i + 1), n))\n",
    "    clf = MultinomialNB()\n",
    "    clf.fit(x[tr], y[tr])\n",
    "    y_pred_va = clf.predict_proba(x[va])\n",
    "    y_pred_te = clf.predict_proba(x_te)\n",
    "    util.log('va acc:%f' % micro_avg_f1(y[va], clf.predict(x[va])))\n",
    "    util.log('te acc:%f' % micro_avg_f1(y_te, clf.predict(x_te)))\n",
    "    score_va += micro_avg_f1(y[va], clf.predict(x[va]))\n",
    "    score_te += micro_avg_f1(y_te, clf.predict(x_te))\n",
    "    stack[va] += y_pred_va\n",
    "    stack_te += y_pred_te\n",
    "score_va /= n\n",
    "score_te /= n\n",
    "util.log('va avg acc:%f' % score_va)\n",
    "util.log('te avg acc:%f' % score_te)\n",
    "stack_te /= n\n",
    "stack_all = np.vstack([stack, stack_te])\n",
    "df_stack = pd.DataFrame(index=range(len(df_all)))\n",
    "for i in range(stack_all.shape[1]):\n",
    "    df_stack['tfidf_mnb_{}'.format(i)] = stack_all[:, i]\n",
    "\n",
    "df_stack.to_csv(param.data_path + '/output/feature/tfidf/mnb_prob_12w.csv', index=None, encoding='utf8')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn import svm\n",
    "\n",
    "\n",
    "############################ svc stack ############################\n",
    "tr_num = param.cv_train_num\n",
    "num_class = len(pd.value_counts(df_all['penalty']))\n",
    "n = 5\n",
    "\n",
    "x = x_sp[:tr_num]\n",
    "y = df_all['penalty'][:tr_num]\n",
    "x_te = x_sp[tr_num:]\n",
    "y_te = df_all['penalty'][tr_num:]\n",
    "\n",
    "stack = np.zeros((x.shape[0], num_class))\n",
    "stack_te = np.zeros((x_te.shape[0], num_class))\n",
    "\n",
    "score_va = 0\n",
    "score_te = 0\n",
    "for i, (tr, va) in enumerate(StratifiedKFold(y, n_folds=n, random_state=param.seed)):\n",
    "    util.log('stack:%d/%d' % ((i + 1), n))\n",
    "    clf = svm.LinearSVC(loss='hinge', tol=0.000001, C=0.5, verbose=1, random_state=param.seed, max_iter=5000)\n",
    "    clf.fit(x[tr], y[tr])\n",
    "    y_pred_va = clf.decision_function(x[va])\n",
    "    y_pred_te = clf.decision_function(x_te)\n",
    "    util.log('va acc:%f' % micro_avg_f1(y[va], clf.predict(x[va])))\n",
    "    util.log('te acc:%f' % micro_avg_f1(y_te, clf.predict(x_te)))\n",
    "    score_va += micro_avg_f1(y[va], clf.predict(x[va]))\n",
    "    score_te += micro_avg_f1(y_te, clf.predict(x_te))\n",
    "    stack[va] += y_pred_va\n",
    "    stack_te += y_pred_te\n",
    "score_va /= n\n",
    "score_te /= n\n",
    "util.log('va avg acc:%f' % score_va)\n",
    "util.log('te avg acc:%f' % score_te)\n",
    "stack_te /= n\n",
    "stack_all = np.vstack([stack, stack_te])\n",
    "df_stack = pd.DataFrame(index=range(len(df_all)))\n",
    "for i in range(stack_all.shape[1]):\n",
    "    df_stack['tfidf_svc_{}'.format(i)] = stack_all[:, i]\n",
    "\n",
    "df_stack.to_csv(param.data_path + '/output/feature/tfidf/svc_prob_12w.csv', index=None, encoding='utf8')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# coding=utf-8\n",
    "import re\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "import param\n",
    "import util\n",
    "\n",
    "df_tr = []\n",
    "util.log('For train.txt:')\n",
    "for i,line in enumerate(open(param.data_path + '/input/train.txt')):\n",
    "    if i % 1000 == 1:\n",
    "        util.log('iter = %d' % i)\n",
    "    segs = line.split('\\t')\n",
    "    row = {}\n",
    "    row['id'] = segs[0]\n",
    "    row['raw_content'] = segs[1].strip()\n",
    "    df_tr.append(row)\n",
    "df_tr = pd.DataFrame(df_tr)\n",
    "\n",
    "# df_te = []\n",
    "# util.log('For test.txt:')\n",
    "# for i,line in enumerate(open(param.data_path + '/input/test.txt')):\n",
    "#     if i % 1000 == 1:\n",
    "#         util.log('iter = %d' % i)\n",
    "#     segs = line.split('\\t')\n",
    "#     row = {}\n",
    "#     row['id'] = segs[0]\n",
    "#     row['raw_content'] = segs[1].strip()\n",
    "#     df_te.append(row)\n",
    "# df_te = pd.DataFrame(df_te)\n",
    "\n",
    "df_all = pd.concat([df_tr]).reset_index(drop=True)\n",
    "\n",
    "amt_list = []\n",
    "for i, row in df_all.iterrows():\n",
    "    if i % 1000 == 1:\n",
    "        util.log('iter = %d' % i)\n",
    "    amt = re.findall(u'(\\d*\\.?\\d+)元', row['raw_content'].decode('utf8'))\n",
    "    amt_tt = re.findall(u'(\\d*\\.?\\d+)万元', row['raw_content'].decode('utf8'))\n",
    "    for a in amt:\n",
    "        amt_list.append([row['id'], float(a)])\n",
    "    for a in amt_tt:\n",
    "        amt_list.append([row['id'], float(a) * 10000])\n",
    "amt_feat = pd.DataFrame(amt_list, columns=['id', 'amount'])\n",
    "amt_feat = amt_feat.groupby('id')['amount'].agg([sum, min, max, np.ptp, np.mean, np.std]).reset_index()\n",
    "amt_feat = pd.merge(df_all, amt_feat, how='left', on='id').drop(['id', 'raw_content'], axis=1)\n",
    "amt_feat.columns = ['amt_' + i for i in amt_feat.columns]\n",
    "\n",
    "amt_feat.to_csv(param.data_path + '/output/feature/amt/amt_12w.csv', index=None, encoding='utf8')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# coding=utf-8\n",
    "import codecs\n",
    "import subprocess\n",
    "from collections import namedtuple\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from gensim.models import Doc2Vec\n",
    "from sklearn.cross_validation import cross_val_score\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "\n",
    "import param\n",
    "import util\n",
    "\n",
    "############################ 加载数据 ############################\n",
    "df_all = pd.read_csv(param.data_path + '/output/corpus/all_data.csv', encoding='utf8', nrows=param.train_num).reset_index()\n",
    "df_all['penalty'] = df_all['penalty'] - 1\n",
    "\n",
    "\n",
    "############################ 定义函数、类及变量 ############################\n",
    "def run_cmd(cmd):\n",
    "    print(cmd)\n",
    "    process = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)\n",
    "    for t, line in enumerate(iter(process.stdout.readline, b'')):\n",
    "        line = line.decode('utf8').rstrip()\n",
    "        print(line)\n",
    "    process.communicate()\n",
    "    return process.returncode\n",
    "\n",
    "\n",
    "SentimentDocument = namedtuple('SentimentDocument', 'words tags')\n",
    "\n",
    "\n",
    "class Doc_list(object):\n",
    "    def __init__(self, f):\n",
    "        self.f = f\n",
    "    def __iter__(self):\n",
    "        for i,line in enumerate(codecs.open(self.f,encoding='utf8')):\n",
    "            words = line.strip().split(' ')\n",
    "            tags = [int(words[0][2:])]\n",
    "            words = words[1:]\n",
    "            yield SentimentDocument(words,tags)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "############################ 准备数据 ############################\n",
    "doc_f = codecs.open(param.data_path + '/output/corpus/doc_for_d2v_12w.txt', 'w', encoding='utf8')\n",
    "for i, contents in enumerate(df_all.iloc[:param.train_num]['content']):\n",
    "    words = []\n",
    "    for word in contents.split(' '):\n",
    "        words.append(word)\n",
    "    tags = [i]\n",
    "    if i % 10000 == 0:\n",
    "        util.log('iter = %d' % i)\n",
    "    doc_f.write(u'_*{} {}\\n'.format(i, ' '.join(words)))\n",
    "doc_f.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "############################ dbow d2v ############################\n",
    "d2v = Doc2Vec(dm=0, size=300, negative=5, hs=0, min_count=3, window=30, sample=1e-5, workers=8, alpha=0.025, min_alpha=0.025)\n",
    "doc_list = Doc_list(param.data_path + '/output/corpus/doc_for_d2v_12w.txt')\n",
    "d2v.build_vocab(doc_list)\n",
    "\n",
    "df_lb = df_all['penalty']\n",
    "\n",
    "for i in range(5):\n",
    "    util.log('pass: ' + str(i))\n",
    "    #     run_cmd('shuf alldata-id.txt > alldata-id-shuf.txt')\n",
    "    doc_list = Doc_list(param.data_path + '/output/corpus/doc_for_d2v_12w.txt')\n",
    "    d2v.train(doc_list, total_examples=d2v.corpus_count, epochs=d2v.iter)\n",
    "    X_d2v = np.array([d2v.docvecs[i] for i in range(param.train_num)])\n",
    "    scores = cross_val_score(LogisticRegression(C=3), X_d2v, df_lb, cv=5)\n",
    "    util.log('dbow: ' + str(scores) + ' ' + str(np.mean(scores)))\n",
    "d2v.save(param.data_path + '/output/model/dbow_d2v_12w.model')\n",
    "util.log('Save done!')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "############################ dm d2v ############################\n",
    "d2v = Doc2Vec(dm=1, size=300, negative=5, hs=0, min_count=3, window=30, sample=1e-5, workers=8, alpha=0.025, min_alpha=0.025)\n",
    "doc_list = Doc_list(param.data_path + '/output/corpus/doc_for_d2v_12w.txt')\n",
    "d2v.build_vocab(doc_list)\n",
    "\n",
    "df_lb = df_all['penalty']\n",
    "\n",
    "for i in range(10):\n",
    "    util.log('pass: ' + str(i))\n",
    "    #     run_cmd('shuf alldata-id.txt > alldata-id-shuf.txt')\n",
    "    doc_list = Doc_list(param.data_path + '/output/corpus/doc_for_d2v_12w.txt')\n",
    "    d2v.train(doc_list, total_examples=d2v.corpus_count, epochs=d2v.iter)\n",
    "    X_d2v = np.array([d2v.docvecs[i] for i in range(param.train_num)])\n",
    "    scores = cross_val_score(LogisticRegression(C=3), X_d2v, df_lb, cv=5)\n",
    "    util.log('dm: ' + str(scores) + ' ' + str(np.mean(scores)))\n",
    "d2v.save(param.data_path + '/output/model/dm_d2v_12w.model')\n",
    "util.log('Save done!')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# coding=utf-8\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from gensim.models import Doc2Vec\n",
    "from keras.layers.core import Dense, Dropout, Activation\n",
    "from keras.models import Sequential\n",
    "from keras.utils import np_utils\n",
    "from sklearn.cross_validation import StratifiedKFold\n",
    "from sklearn.metrics import f1_score\n",
    "\n",
    "import param\n",
    "import util\n",
    "\n",
    "\n",
    "############################ 定义评估函数 ############################\n",
    "def micro_avg_f1(y_true, y_pred):\n",
    "    return f1_score(y_true, y_pred, average='micro')\n",
    "\n",
    "\n",
    "############################ 加载数据 ############################\n",
    "df_all = pd.read_csv(param.data_path + '/output/corpus/all_data.csv', encoding='utf8', nrows=param.train_num)\n",
    "df_all['penalty'] = df_all['penalty'] - 1\n",
    "\n",
    "model = Doc2Vec.load(param.data_path + '/output/model/dbow_d2v_12w.model')\n",
    "x_sp = np.array([model.docvecs[i] for i in range(param.train_num)])\n",
    "\n",
    "############################ dbowd2v stack ############################\n",
    "np.random.seed(param.seed) # 固定种子，方便复现\n",
    "df_stack = pd.DataFrame(index=range(len(df_all)))\n",
    "tr_num = param.cv_train_num\n",
    "num_class = len(pd.value_counts(df_all['penalty']))\n",
    "n = 5\n",
    "\n",
    "x = x_sp[:tr_num]\n",
    "y = df_all['penalty'][:tr_num]\n",
    "x_te = x_sp[tr_num:]\n",
    "y_te = df_all['penalty'][tr_num:]\n",
    "\n",
    "feat = 'dbowd2v'\n",
    "stack = np.zeros((x.shape[0], num_class))\n",
    "stack_te = np.zeros((x_te.shape[0], num_class))\n",
    "\n",
    "score_va = 0\n",
    "score_te = 0\n",
    "for i, (tr, va) in enumerate(StratifiedKFold(y, n_folds=n, random_state=param.seed)):\n",
    "    util.log('stack:%d/%d' % ((i + 1), n))\n",
    "    y_train = np_utils.to_categorical(y[tr], num_class)\n",
    "    y_test = np_utils.to_categorical(y_te, num_class)\n",
    "    model = Sequential()\n",
    "    model.add(Dense(300, input_shape=(x[tr].shape[1],)))\n",
    "    model.add(Dropout(0.1))\n",
    "    model.add(Activation('tanh'))\n",
    "    model.add(Dense(num_class))\n",
    "    model.add(Activation('softmax'))\n",
    "    model.compile(loss='categorical_crossentropy',\n",
    "                  optimizer='adadelta',\n",
    "                  metrics=['accuracy'])\n",
    "    history = model.fit(x[tr], y_train, shuffle=True,\n",
    "                        batch_size=128, nb_epoch=35,\n",
    "                        verbose=2, validation_data=(x_te, y_test))\n",
    "    y_pred_va = model.predict_proba(x[va])\n",
    "    y_pred_te = model.predict_proba(x_te)\n",
    "    util.log('va acc:%f' % micro_avg_f1(y[va], model.predict_classes(x[va])))\n",
    "    util.log('te acc:%f' % micro_avg_f1(y_te, model.predict_classes(x_te)))\n",
    "    score_va += micro_avg_f1(y[va], model.predict_classes(x[va]))\n",
    "    score_te += micro_avg_f1(y_te, model.predict_classes(x_te))\n",
    "    stack[va] += y_pred_va\n",
    "    stack_te += y_pred_te\n",
    "score_va /= n\n",
    "score_te /= n\n",
    "util.log('va avg acc:%f' % score_va)\n",
    "util.log('te avg acc:%f' % score_te)\n",
    "stack_te /= n\n",
    "stack_all = np.vstack([stack, stack_te])\n",
    "for l in range(stack_all.shape[1]):\n",
    "    df_stack['{}_{}'.format(feat, l)] = stack_all[:, l]\n",
    "\n",
    "df_stack.to_csv(param.data_path + '/output/feature/dbowd2v/nn_prob_12w.csv', encoding='utf8', index=None)\n",
    "util.log('Save dbowd2v stack done!')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# coding=utf-8\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from gensim.models import Doc2Vec\n",
    "from keras.layers.core import Dense, Dropout, Activation\n",
    "from keras.models import Sequential\n",
    "from keras.utils import np_utils\n",
    "from sklearn.cross_validation import StratifiedKFold\n",
    "from sklearn.metrics import f1_score\n",
    "\n",
    "import param\n",
    "import util\n",
    "\n",
    "\n",
    "############################ 定义评估函数 ############################\n",
    "def micro_avg_f1(y_true, y_pred):\n",
    "    return f1_score(y_true, y_pred, average='micro')\n",
    "\n",
    "\n",
    "############################ 加载数据 ############################\n",
    "df_all = pd.read_csv(param.data_path + '/output/corpus/all_data.csv', encoding='utf8', nrows=param.train_num)\n",
    "df_all['penalty'] = df_all['penalty'] - 1\n",
    "\n",
    "model = Doc2Vec.load(param.data_path + '/output/model/dm_d2v_12w.model')\n",
    "x_sp = np.array([model.docvecs[i] for i in range(param.train_num)])\n",
    "\n",
    "############################ dbowd2v stack ############################\n",
    "np.random.seed(param.seed) # 固定种子，方便复现\n",
    "df_stack = pd.DataFrame(index=range(len(df_all)))\n",
    "tr_num = param.cv_train_num\n",
    "num_class = len(pd.value_counts(df_all['penalty']))\n",
    "n = 5\n",
    "\n",
    "x = x_sp[:tr_num]\n",
    "y = df_all['penalty'][:tr_num]\n",
    "x_te = x_sp[tr_num:]\n",
    "y_te = df_all['penalty'][tr_num:]\n",
    "\n",
    "feat = 'dmd2v'\n",
    "stack = np.zeros((x.shape[0], num_class))\n",
    "stack_te = np.zeros((x_te.shape[0], num_class))\n",
    "\n",
    "score_va = 0\n",
    "score_te = 0\n",
    "for i, (tr, va) in enumerate(StratifiedKFold(y, n_folds=n, random_state=param.seed)):\n",
    "    util.log('stack:%d/%d' % ((i + 1), n))\n",
    "    y_train = np_utils.to_categorical(y[tr], num_class)\n",
    "    y_test = np_utils.to_categorical(y_te, num_class)\n",
    "    model = Sequential()\n",
    "    model.add(Dense(300, input_shape=(x[tr].shape[1],)))\n",
    "    model.add(Dropout(0.1))\n",
    "    model.add(Activation('tanh'))\n",
    "    model.add(Dense(num_class))\n",
    "    model.add(Activation('softmax'))\n",
    "    model.compile(loss='categorical_crossentropy',\n",
    "                  optimizer='adadelta',\n",
    "                  metrics=['accuracy'])\n",
    "    history = model.fit(x[tr], y_train, shuffle=True,\n",
    "                        batch_size=128, nb_epoch=35,\n",
    "                        verbose=2, validation_data=(x_te, y_test))\n",
    "    y_pred_va = model.predict_proba(x[va])\n",
    "    y_pred_te = model.predict_proba(x_te)\n",
    "    util.log('va acc:%f' % micro_avg_f1(y[va], model.predict_classes(x[va])))\n",
    "    util.log('te acc:%f' % micro_avg_f1(y_te, model.predict_classes(x_te)))\n",
    "    score_va += micro_avg_f1(y[va], model.predict_classes(x[va]))\n",
    "    score_te += micro_avg_f1(y_te, model.predict_classes(x_te))\n",
    "    stack[va] += y_pred_va\n",
    "    stack_te += y_pred_te\n",
    "score_va /= n\n",
    "score_te /= n\n",
    "util.log('va avg acc:%f' % score_va)\n",
    "util.log('te avg acc:%f' % score_te)\n",
    "stack_te /= n\n",
    "stack_all = np.vstack([stack, stack_te])\n",
    "for l in range(stack_all.shape[1]):\n",
    "    df_stack['{}_{}'.format(feat, l)] = stack_all[:, l]\n",
    "\n",
    "df_stack.to_csv(param.data_path + '/output/feature/dmd2v/nn_prob_12w.csv', encoding='utf8', index=None)\n",
    "util.log('Save dmd2v stack done!')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# coding=utf-8\n",
    "from collections import defaultdict\n",
    "\n",
    "import pandas as pd\n",
    "import param\n",
    "import util\n",
    "from gensim.models import Word2Vec\n",
    "\n",
    "############################ 加载数据 ############################\n",
    "df_all = pd.read_csv(param.data_path + '/output/corpus/all_data.csv', encoding='utf8', nrows=param.train_num)\n",
    "df_all['penalty'] = df_all['penalty'] - 1\n",
    "\n",
    "############################ w2v ############################\n",
    "documents = df_all['content'].values\n",
    "util.log('documents number %d' % len(documents))\n",
    "\n",
    "texts = [[word for word in document.split(' ')] for document in documents]\n",
    "frequency = defaultdict(int)\n",
    "for text in texts:\n",
    "    for token in text:\n",
    "        frequency[token] += 1\n",
    "texts = [[token for token in text if frequency[token] >= 5] for text in texts]\n",
    "\n",
    "util.log('Train Model...')\n",
    "w2v = Word2Vec(texts, size=param.w2v_dim, window=5, iter=15, workers=12, seed=param.seed)\n",
    "w2v.save(param.data_path + '/output/model/w2v_12w.model')\n",
    "util.log('Save done!')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# coding=utf-8\n",
    "from collections import defaultdict\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import param\n",
    "import util\n",
    "from gensim.models import Word2Vec\n",
    "\n",
    "############################ 加载数据 & 模型 ############################\n",
    "df_all = pd.read_csv(param.data_path + '/output/corpus/all_data.csv', encoding='utf8', nrows=param.train_num)\n",
    "df_all['penalty'] = df_all['penalty'] - 1\n",
    "documents = df_all['content'].values\n",
    "texts = [[word for word in document.split(' ')] for document in documents]\n",
    "frequency = defaultdict(int)\n",
    "for text in texts:\n",
    "    for token in text:\n",
    "        frequency[token] += 1\n",
    "texts = [[token for token in text if frequency[token] >= 5] for text in texts]\n",
    "\n",
    "model = Word2Vec.load(param.data_path + '/output/model/w2v_12w.model')\n",
    "\n",
    "############################ w2v ############################\n",
    "util.log('Start get w2v feat..')\n",
    "w2v_feat = np.zeros((len(texts), param.w2v_dim))\n",
    "w2v_feat_avg = np.zeros((len(texts), param.w2v_dim))\n",
    "i = 0\n",
    "for line in texts:\n",
    "    num = 0\n",
    "    for word in line:\n",
    "        num += 1\n",
    "        vec = model[word]\n",
    "        w2v_feat[i, :] += vec\n",
    "    w2v_feat_avg[i, :] = w2v_feat[i, :] / num\n",
    "    i += 1\n",
    "    if i % 1000 == 0:\n",
    "        util.log(i)\n",
    "\n",
    "df_w2v = pd.DataFrame(w2v_feat)\n",
    "df_w2v.columns = ['w2v_' + str(i) for i in df_w2v.columns]\n",
    "df_w2v.to_csv(param.data_path + '/output/feature/w2v/w2v_12w.csv', encoding='utf8', index=None)\n",
    "df_w2v_avg = pd.DataFrame(w2v_feat_avg)\n",
    "df_w2v_avg.columns = ['w2v_avg_' + str(i) for i in df_w2v_avg.columns]\n",
    "df_w2v_avg.to_csv(param.data_path + '/output/feature/w2v/w2v_avg_12w.csv', encoding='utf8', index=None)\n",
    "\n",
    "util.log('Save w2v and w2v_avg feat done!')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# coding=utf-8\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import xgboost as xgb\n",
    "from sklearn.metrics import f1_score\n",
    "\n",
    "import param\n",
    "import util\n",
    "\n",
    "\n",
    "############################ 定义评估函数 ############################\n",
    "def micro_avg_f1(preds, dtrain):\n",
    "    y_true = dtrain.get_label()\n",
    "    return 'micro_avg_f1', f1_score(y_true, preds, average='micro')\n",
    "\n",
    "\n",
    "############################ 加载特征 & 标签 ############################\n",
    "df_tfidf_lr = pd.read_csv(param.data_path + '/output/feature/tfidf/lr_prob_12w.csv')\n",
    "df_tfidf_bnb = pd.read_csv(param.data_path + '/output/feature/tfidf/bnb_prob_12w.csv')\n",
    "df_tfidf_mnb = pd.read_csv(param.data_path + '/output/feature/tfidf/mnb_prob_12w.csv')\n",
    "df_tfidf_svc = pd.read_csv(param.data_path + '/output/feature/tfidf/svc_prob_12w.csv')\n",
    "df_amt = pd.read_csv(param.data_path + '/output/feature/amt/amt_12w.csv')\n",
    "df_dbow_nn = pd.read_csv(param.data_path + '/output/feature/dbowd2v/nn_prob_12w.csv')\n",
    "df_dm_nn = pd.read_csv(param.data_path + '/output/feature/dmd2v/nn_prob_12w.csv')\n",
    "df_w2v = pd.read_csv(param.data_path + '/output/feature/w2v/w2v_12w.csv')\n",
    "\n",
    "df_lb = pd.read_csv(param.data_path + '/output/corpus/all_data.csv', usecols=['id', 'penalty'], nrows=param.train_num)\n",
    "df_lb['penalty'] = df_lb['penalty'] - 1  # 让标签属于 [0, 8)\n",
    "\n",
    "############################ xgboost ############################\n",
    "tr_num = param.cv_train_num\n",
    "df_sub = pd.DataFrame()\n",
    "df_sub['id'] = df_lb.iloc[tr_num:]['id']\n",
    "seed = param.seed\n",
    "\n",
    "n_trees = 10000\n",
    "esr = 200\n",
    "evals = 20\n",
    "\n",
    "df = pd.concat([df_tfidf_lr, df_tfidf_bnb, df_tfidf_mnb, df_amt, df_dbow_nn, df_w2v], axis=1)\n",
    "print(df.columns)\n",
    "num_class = len(pd.value_counts(df_lb['penalty']))\n",
    "x = df.iloc[:tr_num]\n",
    "y = df_lb['penalty'][:tr_num]\n",
    "x_te = df.iloc[tr_num:]\n",
    "y_te = df_lb['penalty'][tr_num:]\n",
    "\n",
    "max_depth = 7\n",
    "min_child_weight = 1\n",
    "subsample = 0.8\n",
    "colsample_bytree = 0.8\n",
    "gamma = 1\n",
    "lam = 0\n",
    "\n",
    "params = {\n",
    "    'objective': 'multi:softmax',\n",
    "    'booster': 'gbtree',\n",
    "    'stratified': True,\n",
    "    'num_class': num_class,\n",
    "    'max_depth': max_depth,\n",
    "    'min_child_weight': min_child_weight,\n",
    "    'subsample': subsample,\n",
    "    'colsample_bytree': colsample_bytree,\n",
    "#     'gamma': gamma,\n",
    "#     'lambda': lam,\n",
    "    \n",
    "    'eta': 0.02,\n",
    "    'silent': 1,\n",
    "    'seed': seed,\n",
    "}\n",
    "\n",
    "dtrain = xgb.DMatrix(x, y)\n",
    "dvalid = xgb.DMatrix(x_te, y_te)\n",
    "watchlist = [(dtrain, 'train'), (dvalid, 'test')]\n",
    "bst = xgb.train(params, dtrain, n_trees, evals=watchlist, feval=micro_avg_f1, maximize=True,\n",
    "                early_stopping_rounds=esr, verbose_eval=evals)\n",
    "df_sub['penalty'] = (bst.predict(dvalid) + 1).astype(int)\n",
    "\n",
    "df_sub['id'] = df_sub['id'].astype(str)\n",
    "df_sub['laws'] = [[1]] * len(df_sub)\n",
    "df_sub.to_json(param.data_path + '/output/result/val/1209-xgb-tfidf_lr_bnb_mnb+amt+dbow_nn+w2v.json', orient='records', lines=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.14"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}
